import getopt
import os
import sys
import time
import xml.etree.ElementTree as ET
from configparser import ConfigParser

import schedule
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

"""任务脚本，根据任务xml执行对应爬虫，可分为定时任务，和即时任务
执行python manage.py -f <file>：立即执行xml中的爬虫任务
执行python manage.py -t <time> -f <file>: 定期执行xml中的爬虫任务 
"""


def start_spider(path):
    """根据xml中的任务异步爬取"""
    tree = ET.parse(path)
    root = tree.getroot()
    config = root.find('config')
    host = config.find('host').text
    port = config.find('port').text
    username = config.find('username').text
    password = config.find('password').text
    database = config.find('database').text
    image_path = config.find('image_path').text

    print(host, port, username, password, database, image_path)

    tasks = root.findall('task')
    try:
        settings = get_project_settings()
        settings.set('LOG_FILE', None)
        settings.set('IMAGES_STORE', image_path)
        settings.set('HOST', host)
        settings.set('PORT', int(port))
        settings.set('USERNAME', username)
        settings.set('PASSWORD', password)
        settings.set('DATABASE', database)
        crawl = CrawlerProcess(settings)
        """CrawlerProcess可以控制多个Crawler同时进行多种爬虫任务，CrawlerRunner是其父类"""

        for task in tasks:

            keyword = task.find('keyword').text
            trans_key = task.find('trans_key').text
            spider = task.find('spider').text

            print(keyword, trans_key, spider, image_path)

            crawl.crawl('jd', name=keyword, trans_key=trans_key)
        crawl.start()
    except Exception as e:
        # TODO: 添加异常处理，报错反馈
        print(e)


def schedule_task(time_, path):
    """定时任务"""

    schedule.every().day.at(time_str=time_).do(start_spider(path))
    while True:
        schedule.run_pending()
        time.sleep(100)


def main():
    # 运行时获取参数
    argv = sys.argv[1:]
    try:
        opts, args = getopt.getopt(argv, "t:f:", ['time=', 'file='])
    except getopt.GetoptError:
        print('usage: python manage.py [-t <time>] -f <file>')
        sys.exit(2)

    time_ = path = None
    for opt, arg in opts:
        if opt in ('-t', '--time'):
            time_ = arg
        elif opt in ('-f', '--file'):
            path = arg
    if time_ is None and path is not None:
        print('path', path)
        start_spider(path)
        pass
    elif time_ is not None and path is not None:
        """定时任务"""
        # print(tiem)
        schedule_task(time_, path)
        pass
    else:
        print('usage: python manage.py [-t <time>] -f <file>')


if __name__ == "__main__":
    start_spider('task.xml')




